{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import SparkSession\n",
    "from pyspark.sql import SparkSession\n",
    "spark=SparkSession.builder.appName('log_reg').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read the dataset\n",
    "df=spark.read.csv('Log_Reg_dataset.csv',inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 6)\n"
     ]
    }
   ],
   "source": [
    "#check the shape of the data \n",
    "print((df.count(),len(df.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Country: string (nullable = true)\n",
      " |-- Age: integer (nullable = true)\n",
      " |-- Repeat_Visitor: integer (nullable = true)\n",
      " |-- Search_Engine: string (nullable = true)\n",
      " |-- Web_pages_viewed: integer (nullable = true)\n",
      " |-- Status: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#printSchema\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Country',\n",
       " 'Age',\n",
       " 'Repeat_Visitor',\n",
       " 'Search_Engine',\n",
       " 'Web_pages_viewed',\n",
       " 'Status']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of columns in dataset\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+--------------+-------------+----------------+------+\n",
      "|  Country|Age|Repeat_Visitor|Search_Engine|Web_pages_viewed|Status|\n",
      "+---------+---+--------------+-------------+----------------+------+\n",
      "|    India| 41|             1|        Yahoo|              21|     1|\n",
      "|   Brazil| 28|             1|        Yahoo|               5|     0|\n",
      "|   Brazil| 40|             0|       Google|               3|     0|\n",
      "|Indonesia| 31|             1|         Bing|              15|     1|\n",
      "| Malaysia| 32|             0|       Google|              15|     1|\n",
      "+---------+---+--------------+-------------+----------------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#view the dataset\n",
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------+-----------------+-----------------+-------------+-----------------+------------------+\n",
      "|summary| Country|              Age|   Repeat_Visitor|Search_Engine| Web_pages_viewed|            Status|\n",
      "+-------+--------+-----------------+-----------------+-------------+-----------------+------------------+\n",
      "|  count|   20000|            20000|            20000|        20000|            20000|             20000|\n",
      "|   mean|    null|         28.53955|           0.5029|         null|           9.5533|               0.5|\n",
      "| stddev|    null|7.888912950773227|0.500004090187782|         null|6.073903499824976|0.5000125004687693|\n",
      "|    min|  Brazil|               17|                0|         Bing|                1|                 0|\n",
      "|    max|Malaysia|              111|                1|        Yahoo|               29|                 1|\n",
      "+-------+--------+-----------------+-----------------+-------------+-----------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Exploratory Data Analysis\n",
    "df.describe().show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----+\n",
      "|  Country|count|\n",
      "+---------+-----+\n",
      "| Malaysia| 1218|\n",
      "|    India| 4018|\n",
      "|Indonesia|12178|\n",
      "|   Brazil| 2586|\n",
      "+---------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Country').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-----+\n",
      "|Search_Engine|count|\n",
      "+-------------+-----+\n",
      "|        Yahoo| 9859|\n",
      "|         Bing| 4360|\n",
      "|       Google| 5781|\n",
      "+-------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Search_Engine').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-----+\n",
      "|Status|count|\n",
      "+------+-----+\n",
      "|     1|10000|\n",
      "|     0|10000|\n",
      "+------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Status').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------------------+-------------------+---------------------+--------------------+\n",
      "|  Country|          avg(Age)|avg(Repeat_Visitor)|avg(Web_pages_viewed)|         avg(Status)|\n",
      "+---------+------------------+-------------------+---------------------+--------------------+\n",
      "| Malaysia|27.792282430213465| 0.5730706075533661|   11.192118226600986|  0.6568144499178982|\n",
      "|    India|27.976854156296664| 0.5433051269288203|   10.727227476356397|  0.6212045793927327|\n",
      "|Indonesia| 28.43159796354081| 0.5207751683363442|    9.985711939563148|  0.5422893742814913|\n",
      "|   Brazil|30.274168600154677|  0.322892498066512|    4.921113689095128|0.038669760247486466|\n",
      "+---------+------------------+-------------------+---------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Country').mean().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+------------------+-------------------+---------------------+------------------+\n",
      "|Search_Engine|          avg(Age)|avg(Repeat_Visitor)|avg(Web_pages_viewed)|       avg(Status)|\n",
      "+-------------+------------------+-------------------+---------------------+------------------+\n",
      "|        Yahoo|28.569226087838523| 0.5094837204584644|    9.599655137437875|0.5071508266558474|\n",
      "|         Bing| 28.68394495412844| 0.4720183486238532|    9.114908256880733|0.4559633027522936|\n",
      "|       Google|28.380038055699707| 0.5149628092025601|    9.804878048780488|0.5210171250648676|\n",
      "+-------------+------------------+-------------------+---------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Search_Engine').mean().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------+-------------------+---------------------+-----------+\n",
      "|Status|avg(Age)|avg(Repeat_Visitor)|avg(Web_pages_viewed)|avg(Status)|\n",
      "+------+--------+-------------------+---------------------+-----------+\n",
      "|     1| 26.5435|             0.7019|              14.5617|        1.0|\n",
      "|     0| 30.5356|             0.3039|               4.5449|        0.0|\n",
      "+------+--------+-------------------+---------------------+-----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Status').mean().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting categorical data to numerical form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import required libraries\n",
    "\n",
    "from pyspark.ml.feature import StringIndexer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Indexing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_engine_indexer = StringIndexer(inputCol=\"Search_Engine\", outputCol=\"Search_Engine_Num\").fit(df)\n",
    "df = search_engine_indexer.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+--------------+-------------+----------------+------+-----------------+\n",
      "|Country|Age|Repeat_Visitor|Search_Engine|Web_pages_viewed|Status|Search_Engine_Num|\n",
      "+-------+---+--------------+-------------+----------------+------+-----------------+\n",
      "|India  |41 |1             |Yahoo        |21              |1     |0.0              |\n",
      "|Brazil |28 |1             |Yahoo        |5               |0     |0.0              |\n",
      "|Brazil |40 |0             |Google       |3               |0     |1.0              |\n",
      "+-------+---+--------------+-------------+----------------+------+-----------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(3,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#one hot encoding\n",
    "search_engine_encoder = OneHotEncoder(inputCol=\"Search_Engine_Num\", outputCol=\"Search_Engine_Vector\")\n",
    "df = search_engine_encoder.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+--------------+-------------+----------------+------+-----------------+--------------------+\n",
      "|Country|Age|Repeat_Visitor|Search_Engine|Web_pages_viewed|Status|Search_Engine_Num|Search_Engine_Vector|\n",
      "+-------+---+--------------+-------------+----------------+------+-----------------+--------------------+\n",
      "|India  |41 |1             |Yahoo        |21              |1     |0.0              |(2,[0],[1.0])       |\n",
      "|Brazil |28 |1             |Yahoo        |5               |0     |0.0              |(2,[0],[1.0])       |\n",
      "|Brazil |40 |0             |Google       |3               |0     |1.0              |(2,[1],[1.0])       |\n",
      "+-------+---+--------------+-------------+----------------+------+-----------------+--------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(3,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-----+\n",
      "|Search_Engine|count|\n",
      "+-------------+-----+\n",
      "|Yahoo        |9859 |\n",
      "|Google       |5781 |\n",
      "|Bing         |4360 |\n",
      "+-------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Search_Engine').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+-----+\n",
      "|Search_Engine_Num|count|\n",
      "+-----------------+-----+\n",
      "|0.0              |9859 |\n",
      "|1.0              |5781 |\n",
      "|2.0              |4360 |\n",
      "+-----------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Search_Engine_Num').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|Search_Engine_Vector|count|\n",
      "+--------------------+-----+\n",
      "|(2,[0],[1.0])       |9859 |\n",
      "|(2,[1],[1.0])       |5781 |\n",
      "|(2,[],[])           |4360 |\n",
      "+--------------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Search_Engine_Vector').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "country_indexer = StringIndexer(inputCol=\"Country\", outputCol=\"Country_Num\").fit(df)\n",
    "df = country_indexer.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------+\n",
      "|Country|Country_Num|\n",
      "+-------+-----------+\n",
      "|India  |1.0        |\n",
      "|Brazil |2.0        |\n",
      "|Brazil |2.0        |\n",
      "+-------+-----------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select(['Country','Country_Num']).show(3,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#one hot encoding\n",
    "country_encoder = OneHotEncoder(inputCol=\"Country_Num\", outputCol=\"Country_Vector\")\n",
    "df = country_encoder.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------+--------------+\n",
      "|Country|country_Num|Country_Vector|\n",
      "+-------+-----------+--------------+\n",
      "|India  |1.0        |(3,[1],[1.0]) |\n",
      "|Brazil |2.0        |(3,[2],[1.0]) |\n",
      "|Brazil |2.0        |(3,[2],[1.0]) |\n",
      "+-------+-----------+--------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select(['Country','country_Num','Country_Vector']).show(3,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----+\n",
      "|Country  |count|\n",
      "+---------+-----+\n",
      "|Indonesia|12178|\n",
      "|India    |4018 |\n",
      "|Brazil   |2586 |\n",
      "|Malaysia |1218 |\n",
      "+---------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Country').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-----+\n",
      "|Country_Num|count|\n",
      "+-----------+-----+\n",
      "|0.0        |12178|\n",
      "|1.0        |4018 |\n",
      "|2.0        |2586 |\n",
      "|3.0        |1218 |\n",
      "+-----------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Country_Num').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-----+\n",
      "|Country_Vector|count|\n",
      "+--------------+-----+\n",
      "|(3,[0],[1.0]) |12178|\n",
      "|(3,[1],[1.0]) |4018 |\n",
      "|(3,[2],[1.0]) |2586 |\n",
      "|(3,[],[])     |1218 |\n",
      "+--------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy('Country_Vector').count().orderBy('count',ascending=False).show(5,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_assembler = VectorAssembler(inputCols=['platform_vector','country_vector','Age', 'Repeat_Visitor','Web_pages_viewed'], outputCol=\"features\")\n",
    "df = df_assembler.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Country: string (nullable = true)\n",
      " |-- Age: integer (nullable = true)\n",
      " |-- Repeat_Visitor: integer (nullable = true)\n",
      " |-- Platform: string (nullable = true)\n",
      " |-- Web_pages_viewed: integer (nullable = true)\n",
      " |-- Status: integer (nullable = true)\n",
      " |-- platform_num: double (nullable = false)\n",
      " |-- platform_vector: vector (nullable = true)\n",
      " |-- country_num: double (nullable = false)\n",
      " |-- country_vector: vector (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------+------+\n",
      "|features                           |Status|\n",
      "+-----------------------------------+------+\n",
      "|[1.0,0.0,0.0,1.0,0.0,41.0,1.0,21.0]|1     |\n",
      "|[1.0,0.0,0.0,0.0,1.0,28.0,1.0,5.0] |0     |\n",
      "|(8,[1,4,5,7],[1.0,1.0,40.0,3.0])   |0     |\n",
      "|(8,[2,5,6,7],[1.0,31.0,1.0,15.0])  |1     |\n",
      "|(8,[1,5,7],[1.0,32.0,15.0])        |1     |\n",
      "|(8,[1,4,5,7],[1.0,1.0,32.0,3.0])   |0     |\n",
      "|(8,[1,4,5,7],[1.0,1.0,32.0,6.0])   |0     |\n",
      "|(8,[1,2,5,7],[1.0,1.0,27.0,9.0])   |0     |\n",
      "|(8,[0,2,5,7],[1.0,1.0,32.0,2.0])   |0     |\n",
      "|(8,[2,5,6,7],[1.0,31.0,1.0,16.0])  |1     |\n",
      "+-----------------------------------+------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select(['features','Status']).show(10,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "#select data for building model\n",
    "model_df=df.select(['features','Status'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split the data \n",
    "training_df,test_df=model_df.randomSplit([0.75,0.25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14907"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-----+\n",
      "|Status|count|\n",
      "+------+-----+\n",
      "|     1| 7417|\n",
      "|     0| 7490|\n",
      "+------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training_df.groupBy('Status').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5093"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-----+\n",
      "|Status|count|\n",
      "+------+-----+\n",
      "|     1| 2583|\n",
      "|     0| 2510|\n",
      "+------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_df.groupBy('Status').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_reg=LogisticRegression(labelCol='Status').fit(training_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_results=log_reg.evaluate(training_df).predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+----------------------------------------+\n",
      "|Status|prediction|probability                             |\n",
      "+------+----------+----------------------------------------+\n",
      "|1     |1.0       |[0.2978572628475072,0.7021427371524929] |\n",
      "|1     |1.0       |[0.2978572628475072,0.7021427371524929] |\n",
      "|1     |1.0       |[0.16704676975730415,0.8329532302426959]|\n",
      "|1     |1.0       |[0.16704676975730415,0.8329532302426959]|\n",
      "|1     |1.0       |[0.16704676975730415,0.8329532302426959]|\n",
      "|1     |1.0       |[0.08659913656062515,0.9134008634393749]|\n",
      "|1     |1.0       |[0.08659913656062515,0.9134008634393749]|\n",
      "|1     |1.0       |[0.08659913656062515,0.9134008634393749]|\n",
      "|1     |1.0       |[0.08659913656062515,0.9134008634393749]|\n",
      "|1     |1.0       |[0.08659913656062515,0.9134008634393749]|\n",
      "+------+----------+----------------------------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_results.filter(train_results['Status']==1).filter(train_results['prediction']==1).select(['Status','prediction','probability']).show(10,False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Probability at 0 index is for 0 class and probabilty as 1 index is for 1 class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_preds=train_results.filter(train_results['Status']==1).filter(train_results['prediction']==1).count()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7417"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df.filter(training_df['Status']==1).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9366320614803829"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accuracy on training dataset \n",
    "float(correct_preds)/(training_df.filter(training_df['Status']==1).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test Set results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "results=log_reg.evaluate(test_df).predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+\n",
      "|Status|prediction|\n",
      "+------+----------+\n",
      "|0     |0.0       |\n",
      "|0     |0.0       |\n",
      "|0     |0.0       |\n",
      "|0     |0.0       |\n",
      "|1     |0.0       |\n",
      "|0     |0.0       |\n",
      "|1     |1.0       |\n",
      "|0     |1.0       |\n",
      "|1     |1.0       |\n",
      "|1     |1.0       |\n",
      "+------+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results.select(['Status','prediction']).show(10,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- features: vector (nullable = true)\n",
      " |-- Status: integer (nullable = true)\n",
      " |-- rawPrediction: vector (nullable = true)\n",
      " |-- probability: vector (nullable = true)\n",
      " |-- prediction: double (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#confusion matrix\n",
    "true_postives = results[(results.Status == 1) & (results.prediction == 1)].count()\n",
    "true_negatives = results[(results.Status == 0) & (results.prediction == 0)].count()\n",
    "false_positives = results[(results.Status == 0) & (results.prediction == 1)].count()\n",
    "false_negatives = results[(results.Status == 1) & (results.prediction == 0)].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2356\n",
      "2363\n",
      "158\n",
      "157\n",
      "5034\n",
      "5034\n"
     ]
    }
   ],
   "source": [
    "print (true_postives)\n",
    "print (true_negatives)\n",
    "print (false_positives)\n",
    "print (false_negatives)\n",
    "print(true_postives+true_negatives+false_positives+false_negatives)\n",
    "print (results.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.937524870672503\n"
     ]
    }
   ],
   "source": [
    "recall = float(true_postives)/(true_postives + false_negatives)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9371519490851233\n"
     ]
    }
   ],
   "source": [
    "precision = float(true_postives) / (true_postives + false_positives)\n",
    "print(precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9374255065554231\n"
     ]
    }
   ],
   "source": [
    "accuracy=float((true_postives+true_negatives) /(results.count()))\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
